package com.gxljc.bear.crawler.itaogao.mrjjxw;

import com.gxljc.commons.mongo.cli.BaseMongoRepository;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.jingji21.Jingji21UpdateCrawler;
import com.gxljc.bear.crawler.util.DateUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;

import java.io.Serializable;
import java.text.DecimalFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.LinkedList;
import java.util.List;

/**
 * 每日经济新闻 seed 数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-9-13
 */
public class MrjjxwUpdateCrawler implements Serializable {
    public static Logger LOG = Logger.getLogger(Jingji21UpdateCrawler.class);
    private Boolean proxy = false;
    MrjjxwSeedCrawler crawler = null;

    public MrjjxwUpdateCrawler(Boolean proxy) {
        this.proxy = proxy;
        crawler = new MrjjxwSeedCrawler(proxy);
    }

    public void crawl() throws Exception {
        List<String> seeds = getSeed();
        if (CollectionUtils.isEmpty(seeds)) {
            LOG.error("page is empty");
            return;
        }
        crawlNoSpark(seeds);
    }

    public void crawlNoSpark(List<String> seeds) {
        for (String url : seeds) {
            try {
                crawler.crawl(url);
            } catch (Exception e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }

    //构造URL seed
    public List<String> getSeed() throws Exception {
        Date dateSeed = getDateSeed();
        if (dateSeed == null) return null;
        List<String> urls = new LinkedList<String>();
        final int MAX_NODE = 25;
        String dateFormat = DateUtil.dateFormat(dateSeed, "yyyyMMdd");
        for (int i = 1; i < MAX_NODE; i++) {
            String index = new DecimalFormat("00").format(i);
            String url = String.format(ItaogaoConst.NEWSPAPER_SEED.MRJJXW.getValue(), dateFormat, index);
            urls.add(url);
        }
        return urls;
    }

    //获取从当前到1月1号的列表
    private Date getDateSeed() {
        Calendar now = Calendar.getInstance();
        return now.getTime();
    }
}
